"""
Case Type   : 功能测试
Case Name   : 主节点开启流控备1备2开启实时构建，流控开启，主节点kill掉进程
Create At   : 2025/03/25
Owner       : @chen
Description :
    1.主节点设置流控参数为
gs_guc set -N NODE -c 'recovery_time_target=3';
    2.备1备2开启实时构建
gs_guc set -N NODE1 -c 'recovery_parse_workers=4';
gs_guc set -N NODE1 -c 'recovery_redo_workers=4';
gs_guc set -N NODE1 -c 'host_standby=off';
gs_guc set -N NODE1 -c 'ss_enable_ondemand_recovery=on';
gs_guc set -N NODE1 -c 'ss_enable_ondemand_realtime_build=on';
    重启集群
cm_ctl stop; cm_ctl start
    3.主节点kill掉dn进程
    查看日志信息
    4.主节点kill掉dss进程
    查看日志信息
    5.主节点kill掉cma进程
    查看日志信息
    6.主节点kill掉cms进程
    查看日志信息
    7.恢复环境
Expect      :
    1.设置成功
    2.成功
    3.流控中断，主节点恢复后，流控功能重新开启
    4.流控中断，主节点恢复后，流控功能重新开启
    5.流控中断，主节点恢复后，流控功能重新开启
    6.流控不受影响
    7.成功
History     :
"""

import unittest
import os
import time
from testcase.utils.Constant import Constant
from testcase.utils.CommonSH import CommonSH
from testcase.utils.Common import Common
from testcase.utils.Logger import Logger
from testcase.SS_FUNC.realtime_build_control.Realtime_Build_Control_Utils import rbcu
from yat.test import Node
logger = Logger()

class TestRealtimeBuild_Control0053(unittest.TestCase):
    def setUp(self):
        logger.info(f'--------{os.path.basename(__file__)} start--------')
        self.constant = Constant()
        self.com = Common()
        self.rbcu = rbcu()
        self.pri_node = Node('PrimaryDbUser')
        self.pri_sh = CommonSH('PrimaryDbUser')
        self.sta2_sh = CommonSH('Standby2DbUser')
        self.sta2_node = Node('Standby2DbUser')
        self.sta1_node = Node('Standby1DbUser')
        self.sta1_sh = CommonSH('Standby1DbUser')
        self.recovery_time_target = {}
        self.real_time = {}

        logger.info('------检查数据库状态是否正常------')
        status = self.pri_sh.exec_cm_ctl('query', '-Cv')
        self.assertIn("cluster_state   : Normal", status)
        self.pri_hostname = self.pri_node.sh('hostname').result()
        logger.info(f'主节点hostname: {self.pri_hostname}')
        self.sta1_hostname = self.sta1_node.sh('hostname').result()
        logger.info(f'备1节点hostname: {self.sta1_hostname}')
        self.sta2_hostname = self.sta2_node.sh('hostname').result()
        logger.info(f'备2节点hostname: {self.sta2_hostname}')

    def test_shared_storage(self):

        text = '-----step1:主节点设置流控参数为3 expect:设置成功-----'
        logger.info(text)
        self.recovery_time_target = self.rbcu.set_control(self.pri_hostname, 3, True)

        text = '-----step2:备1备2开启实时构建 expect:设置成功-----'
        logger.info(text)
        self.real_time = self.rbcu.set_real_time(self.sta1_hostname, True)
        self.rbcu.set_real_time(self.sta2_hostname)
        logger.info('-----重启集群-----')
        restart = self.pri_sh.exec_cm_ctl('stop && cm_ctl start')
        logger.info(restart)
        self.assertIn(self.constant.cm_start_success_msg, restart,
                      '执行失败')
        logger.info('-----查看日志-----')
        pres = self.rbcu.control_check(self.pri_node, '6001')
        self.assertIn('enable realtime-build log ctrl', pres, '主节点校验失败')
        self.assertTrue('inst_id: 2' in pres or 'inst_id: 1' in pres, '主节点校验失败')
        ret1 = self.rbcu.send_control_check(self.sta1_node, '6002')
        self.assertIn('send primary node 0', ret1, '主节点校验失败')
        ret2 = self.rbcu.send_control_check(self.sta2_node, '6003')
        self.assertIn('send primary node 0', ret2, '主节点校验失败')
        ret = self.rbcu.query_view(self.pri_sh)
        self.assertIn('(2 rows)', ret, '主节点校验失败')
        ret = self.rbcu.query_view(self.sta1_sh)
        self.assertIn('(0 rows)', ret, '备节点校验失败')
        ret = self.rbcu.query_view(self.sta2_sh)
        self.assertIn('(0 rows)', ret, '备节点校验失败')
        
        text = '-----step3:主节点kill掉dn进程，查看日志 expect:流控重新开启-----'
        logger.info(text)
        pid = self.com.get_pid(self.pri_node, 'bin/gaussdb')
        logger.info(pid)
        cmd = f"kill -9 {pid}"
        logger.info(cmd)
        res = self.pri_node.sh(cmd).result()
        logger.info(res)
        logger.info('-----等待集群恢复-----')
        while True:
            status = self.pri_sh.exec_cm_ctl('query', '-Cv')
            if "cluster_state   : Normal" in status:
                break
            time.sleep(5)
        dnr = self.rbcu.control_check(self.pri_node, '6001')
        self.assertIn('enable realtime-build log ctrl', dnr, '主节点校验失败')
        self.assertTrue('inst_id: 2' in dnr or 'inst_id: 1' in dnr, '主节点校验失败')
        dnr1 = self.rbcu.send_control_check(self.sta1_node, '6002')
        self.assertIn('send primary node 0', dnr1, '主节点校验失败')
        self.assertTrue(ret1 != dnr1)
        dnr2 = self.rbcu.send_control_check(self.sta2_node, '6003')
        self.assertIn('send primary node 0', dnr2, '主节点校验失败')
        self.assertTrue(ret2 != dnr2)
        ret = self.rbcu.query_view(self.pri_sh)
        self.assertIn('(2 rows)', ret, '主节点校验失败')
        ret = self.rbcu.query_view(self.sta1_sh)
        self.assertIn('(0 rows)', ret, '备节点校验失败')
        ret = self.rbcu.query_view(self.sta2_sh)
        self.assertIn('(0 rows)', ret, '备节点校验失败')

        text = '-----step4:主节点kill掉dss进程，查看日志 expect:流控重新开启-----'
        logger.info(text)
        pid = self.com.get_pid(self.pri_node, 'bin/dssserver')
        logger.info(pid)
        cmd = f"kill -9 {pid}"
        logger.info(cmd)
        res = self.pri_node.sh(cmd).result()
        logger.info(res)
        logger.info('-----等待集群恢复-----')
        while True:
            time.sleep(5)
            status = self.pri_sh.exec_cm_ctl('query', '-Cv')
            if "cluster_state   : Normal" in status:
                break
        dssr = self.rbcu.control_check(self.pri_node, '6001')
        self.assertIn('enable realtime-build log ctrl', dssr, '主节点校验失败')
        self.assertTrue('inst_id: 2' in dssr or 'inst_id: 1' in dssr, '主节点校验失败')
        self.assertTrue(dnr != dssr, '流控校验失败')
        dssr1 = self.rbcu.send_control_check(self.sta1_node, '6002')
        self.assertIn('send primary node 0', dssr1, '备节点校验失败')
        self.assertTrue(dssr1 != dnr1)
        dssr2 = self.rbcu.send_control_check(self.sta2_node, '6003')
        self.assertIn('send primary node 0', dssr2, '主节点校验失败')
        self.assertTrue(dssr2 != dnr2)
        ret = self.rbcu.query_view(self.pri_sh)
        self.assertIn('(2 rows)', ret, '主节点校验失败')
        ret = self.rbcu.query_view(self.sta1_sh)
        self.assertIn('(0 rows)', ret, '备节点校验失败')
        ret = self.rbcu.query_view(self.sta2_sh)
        self.assertIn('(0 rows)', ret, '备节点校验失败')

        text = '-----step5:主节点kill掉cma进程，查看日志 expect:流控重新开启-----'
        logger.info(text)
        logger.info('-----kill掉cma进程-----')
        pid = self.com.get_pid(self.pri_node, 'bin/cm_agent')
        logger.info(pid)
        cmd = f"kill -9 {pid}"
        logger.info(cmd)
        res = self.pri_node.sh(cmd).result()
        logger.info(res)
        logger.info('-----等待集群恢复-----')
        while True:
            time.sleep(5)
            status = self.pri_sh.exec_cm_ctl('query', '-Cv')
            if "cluster_state   : Normal" in status:
                break
        cmar = self.rbcu.control_check(self.pri_node, '6001')
        self.assertIn('enable realtime-build log ctrl', cmar, '主节点校验失败')
        self.assertTrue('inst_id: 2' in cmar or 'inst_id: 1' in cmar, '主节点校验失败')
        cmar1 = self.rbcu.send_control_check(self.sta1_node, '6002')
        self.assertIn('send primary node 0', cmar1, '备节点校验失败')
        self.assertTrue(dssr1 == cmar1, '流控校验失败')
        cmar2 = self.rbcu.send_control_check(self.sta2_node, '6003')
        self.assertIn('send primary node 0', cmar2, '主节点校验失败')
        self.assertTrue(dssr2 == cmar2, '流控校验失败')
        ret = self.rbcu.query_view(self.pri_sh)
        self.assertIn('(2 rows)', ret, '主节点校验失败')
        ret = self.rbcu.query_view(self.sta1_sh)
        self.assertIn('(0 rows)', ret, '备节点校验失败')
        ret = self.rbcu.query_view(self.sta2_sh)
        self.assertIn('(0 rows)', ret, '备节点校验失败')

        text = '-----step6:主节点kill掉cms进程，查看日志 expect:流控不受影响-----'
        logger.info(text)
        logger.info('-----kill掉cma进程-----')
        pid = self.com.get_pid(self.pri_node, 'bin/cm_server')
        logger.info(pid)
        cmd = f"kill -9 {pid}"
        logger.info(cmd)
        res = self.pri_node.sh(cmd).result()
        logger.info(res)
        while True:
            time.sleep(5)
            status = self.pri_sh.exec_cm_ctl('query', '-Cv')
            if "cluster_state   : Normal" in status:
                break
        logger.info('-----查看集群状态-----')
        cmsr = self.rbcu.control_check(self.pri_node, '6001')
        self.assertIn('enable realtime-build log ctrl', cmsr, '主节点校验失败')
        self.assertTrue('inst_id: 2' in cmsr or 'inst_id: 1' in cmsr, '主节点校验失败')
        cmsr1 = self.rbcu.send_control_check(self.sta1_node, '6002')
        self.assertIn('send primary node 0', cmsr1, '备节点校验失败')
        self.assertTrue(cmsr1 == cmar1, '流控校验失败')
        cmsr2 = self.rbcu.send_control_check(self.sta2_node, '6003')
        self.assertIn('send primary node 0', cmsr2, '主节点校验失败')
        self.assertTrue(cmsr2 == cmar2, '流控校验失败')
        ret = self.rbcu.query_view(self.pri_sh)
        self.assertIn('(2 rows)', ret, '主节点校验失败')
        ret = self.rbcu.query_view(self.sta1_sh)
        self.assertIn('(0 rows)', ret, '备节点校验失败')
        ret = self.rbcu.query_view(self.sta2_sh)
        self.assertIn('(0 rows)', ret, '备节点校验失败')

    def tearDown(self):
        text = '-----step7:恢复环境; expect:成功-----'
        logger.info(text)
        logger.info('-----主节点恢复-----')
        self.rbcu.restore_guc_param(self.pri_hostname, self.recovery_time_target)
        logger.info('-----备1节点恢复-----')
        self.rbcu.restore_guc_param(self.sta1_hostname, self.real_time)
        logger.info('-----备2节点恢复-----')
        self.rbcu.restore_guc_param(self.sta2_hostname, self.real_time)
        logger.info('-----重启集群-----')
        restart = self.pri_sh.exec_cm_ctl('stop && cm_ctl start')
        logger.info(restart)
        self.assertIn(self.constant.cm_start_success_msg, restart,
                      '执行失败')
        logger.info(f'-----{os.path.basename(__file__)} end-----')
